{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def load_jsonl(filename):\n",
    "    output = []\n",
    "\n",
    "    with open(filename, 'r', encoding=\"utf-8\") as fp:\n",
    "        while jsonl := fp.readline().strip():\n",
    "            jsond = json.loads(jsonl)\n",
    "            output.append(jsond)\n",
    "\n",
    "    return output\n",
    "\n",
    "import csv\n",
    "def dump_csv(data:list, filename:str, header=None):\n",
    "    \"\"\"将列表数据保存到CSV\n",
    "\n",
    "    Args:\n",
    "        data (list): 要保存的二维列表数据\n",
    "        filename (str): 保存的文件名\n",
    "        header (_type_, optional): 添加自定义表头.  默认为None, 数据的第一行作为表头\n",
    "\n",
    "    Returns:\n",
    "        None: return None if success else throw error.\n",
    "    \"\"\"\n",
    "    with open(filename, \"w\", encoding=\"utf-8\", newline='') as fp:  # newline=''解决额外的换行\n",
    "        writer = csv.writer(fp)\n",
    "        data = [header,] + data if header else data\n",
    "        writer.writerows(data)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileExistsError",
     "evalue": "id2rel.txt and id2ent.txt already exists",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileExistsError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[17], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m output_dir \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mdata/Graph/\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m      5\u001b[0m \u001b[39mif\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39misfile(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(output_dir, \u001b[39m\"\u001b[39m\u001b[39mid2rel.txt\u001b[39m\u001b[39m\"\u001b[39m)) \u001b[39mor\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39misfile(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(output_dir, \u001b[39m\"\u001b[39m\u001b[39mid2ent.txt\u001b[39m\u001b[39m\"\u001b[39m)):\n\u001b[1;32m----> 6\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mFileExistsError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mid2rel.txt and id2ent.txt already exists\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m      7\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39misdir(output_dir):\n\u001b[0;32m      8\u001b[0m     os\u001b[39m.\u001b[39mmakedirs(output_dir)\n",
      "\u001b[1;31mFileExistsError\u001b[0m: id2rel.txt and id2ent.txt already exists"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from itertools import chain\n",
    "\n",
    "output_dir = \"data/Graph/\"\n",
    "if os.path.isfile(os.path.join(output_dir, \"id2rel.txt\")) or os.path.isfile(os.path.join(output_dir, \"id2ent.txt\")):\n",
    "    raise FileExistsError(\"id2rel.txt and id2ent.txt already exists\")\n",
    "if not os.path.isdir(output_dir):\n",
    "    os.makedirs(output_dir)\n",
    "\n",
    "train_jsond = load_jsonl(\"data\\CMeIE\\CMeIE_train.jsonl\")\n",
    "dev_jsond  = load_jsonl(\"data\\CMeIE\\CMeIE_dev.jsonl\")\n",
    "all_jsond = train_jsond+dev_jsond\n",
    "\n",
    "entities = set()\n",
    "relations = set()\n",
    "for jsond in all_jsond:\n",
    "    entity_pairs = [(spo[\"subject_type\"]+\":\"+spo[\"subject\"],spo[\"object_type\"][\"@value\"] +\":\"+spo[\"object\"][\"@value\"]) for spo in jsond[\"spo_list\"]]\n",
    "    entities.update(chain(*entity_pairs))\n",
    "    relations.update([spo[\"predicate\"] for spo in jsond[\"spo_list\"]])\n",
    "\n",
    "entities_list = sorted(list(entities))\n",
    "relations_list = sorted(list(relations))\n",
    "\n",
    "with open(os.path.join(output_dir, \"id2ent.txt\"), \"w\", encoding=\"utf-8\") as fp:\n",
    "    fp.write('\\n'.join(entities_list))\n",
    "with open(os.path.join(output_dir, \"id2rel.txt\"), \"w\", encoding=\"utf-8\") as fp:\n",
    "    fp.write('\\n'.join(relations_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ent2id = {k:i for i,k in enumerate(entities_list)}\n",
    "rel2id = {k:i for i,k in enumerate(relations_list)}\n",
    "\n",
    "triples_set = set()\n",
    "for jsond in all_jsond:\n",
    "    for spo in jsond[\"spo_list\"]:\n",
    "        subj = spo[\"subject\"]\n",
    "        subj_type = spo[\"subject_type\"]\n",
    "        rel = spo[\"predicate\"]\n",
    "        obj = spo[\"object\"][\"@value\"]\n",
    "        obj_type = spo[\"object_type\"][\"@value\"]\n",
    "\n",
    "        subji = ent2id[subj_type+\":\"+subj]\n",
    "        reli = rel2id[rel]\n",
    "        obji = ent2id[obj_type+':'+obj]\n",
    "        triples_set.add((obji, reli, subji))\n",
    "triples_list = sorted(list(triples_set))\n",
    "\n",
    "dump_csv(triples_list, os.path.join(output_dir, \"graph.csv\"), header=[\"head\", \"relation\", \"tail\"])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import choice\n",
    "def sample_node_pair(nodes_list:list, nodes_set:set, available_entids:list):\n",
    "    positive = choice(nodes_list)\n",
    "    negative = (positive[0], positive[1], choice(available_entids))\n",
    "    while negative in nodes_set:\n",
    "        negative = (positive[0], positive[1], choice(available_entids))\n",
    "    return positive, negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25237, 32, 8916), (25237, 32, 26082))"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_node_pair(triples_list, list(ent2id.values()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'set' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[67], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m choice({\u001b[39m1\u001b[39;49m,\u001b[39m2\u001b[39;49m,\u001b[39m3\u001b[39;49m})\n",
      "File \u001b[1;32mc:\\Users\\jinnn\\.conda\\envs\\TensorFlow\\lib\\random.py:346\u001b[0m, in \u001b[0;36mRandom.choice\u001b[1;34m(self, seq)\u001b[0m\n\u001b[0;32m    344\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Choose a random element from a non-empty sequence.\"\"\"\u001b[39;00m\n\u001b[0;32m    345\u001b[0m \u001b[39m# raises IndexError if seq is empty\u001b[39;00m\n\u001b[1;32m--> 346\u001b[0m \u001b[39mreturn\u001b[39;00m seq[\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_randbelow(\u001b[39mlen\u001b[39;49m(seq))]\n",
      "\u001b[1;31mTypeError\u001b[0m: 'set' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "choice({1,2,3})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorFlow",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "60ad95de29a2d4f003b4f2410e933e9f54bac822e3bc706f840142e3a87d9714"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
